#include <stdint.h>
#include <immintrin.h>
#include <math.h>
#include "utils.h"
#include <stdlib.h>
#include <stdio.h>

#define L2_IJK_LOOP \
        for (int mb = 0; mb < M; mb += M_CACHE) { \
        for (int nb = 0; nb < N; nb += N_CACHE) { \
        for (int kb = 0; kb < K; kb += K_CACHE) {

#define L2_IKJ_LOOP \
        for (int mb = 0; mb < M; mb += M_CACHE) { \
        for (int kb = 0; kb < K; kb += K_CACHE) { \
        for (int nb = 0; nb < N; nb += N_CACHE) {

#define L2_JKI_LOOP \
        for (int nb = 0; nb < M; nb += N_CACHE) { \
        for (int kb = 0; kb < K; kb += K_CACHE) { \
        for (int mb = 0; mb < N; mb += M_CACHE) {

#define L2_PACK_IJK_LOOP \
       for (int mb = 0; mb < M/M_CACHE; mb++) { \
       for (int nb = 0; nb < N/N_CACHE; nb++) { \
       for (int kb = 0; kb < K/K_CACHE; kb++) {

#define L2_PACK_IKJ_LOOP \
        for (int mb = 0; mb < M/M_CACHE; mb++) { \
        for (int kb = 0; kb < K/K_CACHE; kb++) { \
        for (int nb = 0; nb < N/N_CACHE; nb++) {

#define L2_PACK_JKI_LOOP \
        for (int nb = 0; nb < N/N_CACHE; nb++) { \
        for (int kb = 0; kb < K/K_CACHE; kb++) { \
        for (int mb = 0; mb < M/M_CACHE; mb++) {
    

#define L2_LOOP_END \
        }}}
       

int l2_v2(int niters, int8_t *A, int8_t *B, int32_t *C, size_t M, size_t N, size_t K) {
    for (int i = 0; i < niters; ++i) {
#if L2_LOOP == 1
        L2_IJK_LOOP
#elif L2_LOOP == 2
        L2_IKJ_LOOP
#else
        L2_JKI_LOOP
#endif

                    // L1 loop: [i->j->k], should be ~90% MAC utilization
                    for (int m = mb; m < mb + M_CACHE; m += M_ACC * TILE_M) {
                        for (int n = nb; n < nb + N_CACHE; n += N_ACC * TILE_N) {
                            if (kb == 0) {
                                _tile_zero(TMM4);
                                _tile_zero(TMM5);
                                _tile_zero(TMM6);
                                _tile_zero(TMM7);
                            }
                            else {
                                // Load C00
                                _tile_loadd(TMM4, (int8_t *)(C + m * N + n), N * sizeof(uint32_t));
                                // Load C01
                                _tile_loadd(TMM5, (int8_t *)(C + m * N + (n + TILE_N)), N * sizeof(uint32_t));
                                // Load C10
                                _tile_loadd(TMM6, (int8_t *)(C + (m + TILE_M) * N + n), N * sizeof(uint32_t));
                                // Load C11
                                _tile_loadd(TMM7, (int8_t *)(C + (m + TILE_M) * N + n + TILE_N), N * sizeof(uint32_t));
                                // _tile_stream_loadd(TMM4, (int8_t *)(C + m * N + n), N * sizeof(uint32_t));
                                // _tile_stream_loadd(TMM5, (int8_t *)(C + m * N + (n + TILE_N)), N * sizeof(uint32_t));
                                // _tile_stream_loadd(TMM6, (int8_t *)(C + (m + TILE_M) * N + n), N * sizeof(uint32_t));
                                // _tile_stream_loadd(TMM7, (int8_t *)(C + (m + TILE_M) * N + n + TILE_N), N * sizeof(uint32_t));
                            }

                            for (int k = kb; k < kb + K_CACHE; k += TILE_K) {
                                _tile_loadd(TMM0, A + m * K + k, K);
                                _tile_loadd(TMM1, A + (m + TILE_M) * K + k, K);
                                // _tile_stream_loadd(TMM0, A + m * K + k, K);
                                // _tile_stream_loadd(TMM1, A + (m + TILE_M) * K + k, K);
                                _tile_loadd(TMM2, B + (k / 4) * N * 4 + n * 4, N * 4);
                                _tile_loadd(TMM3, B + (k / 4) * N * 4 + (n + TILE_N) * 4, N * 4);
                                // _tile_stream_loadd(TMM2, B + (k / 4) * N * 4 + n * 4, N * 4);
                                // _tile_stream_loadd(TMM3, B + (k / 4) * N * 4 + (n + TILE_N) * 4, N * 4);

                                _tile_dpbuud(TMM4, TMM0, TMM2);
                                _tile_dpbuud(TMM5, TMM0, TMM3);
                                _tile_dpbuud(TMM6, TMM1, TMM2);
                                _tile_dpbuud(TMM7, TMM1, TMM3);
                            }
                            // Store C00
                            _tile_stored(TMM4, (int8_t *)(C + m * N + n), N * sizeof(int32_t)); 
                            // Store C01
                            _tile_stored(TMM5, (int8_t *)(C + m * N + (n + TILE_N)), N * sizeof(int32_t));
                            // Store C10
                            _tile_stored(TMM6, (int8_t *)(C + (m + TILE_M) * N + n), N * sizeof(int32_t));
                            // Store C11
                            _tile_stored(TMM7, (int8_t *)(C + (m + TILE_M) * N + (n + TILE_N)), N * sizeof(int32_t));
                        }
                    }  // end of L1 loops

        L2_LOOP_END
    }
}

int l2_v2_load(int niters, int8_t *A, int8_t *B, int32_t *C, size_t M, size_t N, size_t K) {
    for (int i = 0; i < niters; ++i) {
#if L2_LOOP == 1
        L2_IJK_LOOP
#elif L2_LOOP == 2
        L2_IKJ_LOOP
#else
        L2_JKI_LOOP
#endif

                    for (int m = mb; m < mb + M_CACHE; m += M_ACC * TILE_M) {
                        for (int n = nb; n < nb + N_CACHE; n += N_ACC * TILE_N) {

                            for (int k = kb; k < kb + K_CACHE; k += TILE_K) {
                                _tile_loadd(TMM0, A + m * K + k, K);
                                _tile_loadd(TMM1, A + (m + TILE_M) * K + k, K);
                                // _tile_loadd(TMM2, B + (k / 4) * N * 4 + n * 4, N * 4);
                                // _tile_loadd(TMM3, B + (k / 4) * N * 4 + (n + TILE_N) * 4, N * 4);
                            }
                        }
                    }

        L2_LOOP_END
    }
}

int l2_v2_with_pack(int niters, int8_t *A, int8_t *B, int32_t *C, size_t M, size_t N, size_t K) {
    for (int i = 0; i < niters; ++i) {
#if L2_LOOP == 1
        L2_PACK_IJK_LOOP
#elif L2_LOOP == 2
        L2_PACK_IKJ_LOOP
#else
        L2_PACK_JKI_LOOP
#endif
                    const size_t A_tile_offset = (mb*(K/K_CACHE)+kb) * (M_CACHE*K_CACHE) * sizeof(uint8_t);
                    const size_t B_tile_offset = (kb*(N/N_CACHE)+nb) * (K_CACHE*N_CACHE) * sizeof(uint8_t); 
                    const size_t C_tile_offset = (mb*(N/N_CACHE)+nb) * (M_CACHE*N_CACHE) * sizeof(uint32_t);
                    // const size_t C_tile_offset = 0;

                    for (int m = 0; m < (M_CACHE/TILE_M); m+=2) {
                        for (int n = 0; n < (N_CACHE/TILE_N); n+=2) {
                            if (0 == kb) {
                                _tile_zero(TMM4);
                                _tile_zero(TMM5);
                                _tile_zero(TMM6);
                                _tile_zero(TMM7);
                            } else {
                                _tile_loadd(TMM4, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                                _tile_loadd(TMM5, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                                _tile_loadd(TMM6, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                                _tile_loadd(TMM7, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t)); 

                                // _tile_stream_loadd(TMM4, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                                // _tile_stream_loadd(TMM5, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                                // _tile_stream_loadd(TMM6, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                                // _tile_stream_loadd(TMM7, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t)); 
                            }

                            for (int k = 0; k < (K_CACHE/TILE_K); k++) {
                                _tile_loadd(TMM0, A + A_tile_offset + m*TILE_M*K_CACHE+k*TILE_K, K_CACHE);
                                _tile_loadd(TMM1, A + A_tile_offset + (m+1)*TILE_M*K_CACHE+k*TILE_K, K_CACHE);
                                _tile_loadd(TMM2, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+n*TILE_N*4, N_CACHE*4);
                                _tile_loadd(TMM3, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+(n+1)*TILE_N*4, N_CACHE*4);
                                // _tile_stream_loadd(TMM2, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+n*TILE_N*4, N_CACHE*4);
                                // _tile_stream_loadd(TMM3, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+(n+1)*TILE_N*4, N_CACHE*4);

                                _tile_dpbuud(TMM4, TMM0, TMM2);
                                _tile_dpbuud(TMM6, TMM1, TMM2);
                                _tile_dpbuud(TMM5, TMM0, TMM3);
                                _tile_dpbuud(TMM7, TMM1, TMM3);
                            }
                            _tile_stored(TMM4, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                            _tile_stored(TMM5, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                            _tile_stored(TMM6, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                            _tile_stored(TMM7, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                        }
                    }  // end of L1 loops

        L2_LOOP_END
    }
}

void prefetch_1024byte_to_l1(const void* base, size_t stride, const int locality){
    // Intel Software Optimization Manual Volume 1 - 9.3.2 Prefetch Instructions (Xeon): 
    // prefetch[t0, t1, t2, nta]
    // https://www.daemon-systems.org/man/__builtin_prefetch.3.html
    // __builtin_prefetch(addr, rw, locality):
    // addr: memory where prefetch wants to prefetch from
    // rw (default is 0): either 0 or 1, corresponding with read and write prefetch, respectively
    // locality (default is 3): 0/1/2/3, the higher the value, the higher the temporal locality in the data
    switch(locality) {
        case 0:  // prefetchnta: L1
            for (size_t i = 0; i < 16; i++){
                __builtin_prefetch(base + i * stride, 0, 0); 
            }
            break;
        case 1:  // prefetcht2: L2
            for (size_t i = 0; i < 16; i++){
                __builtin_prefetch(base + i * stride, 0, 1);
            }
            break;
        case 2:  // prefetcht1: L2
            for (size_t i = 0; i < 16; i++){
                __builtin_prefetch(base + i * stride, 0, 2); 
            }
            break;
        case 3:  // prefetcht0: L1 and L2
            for (size_t i = 0; i < 16; i++){
                __builtin_prefetch(base + i * stride, 0, 3);
            }
            break;
        default:
            break;
    }
}

size_t A_prefetch_block_offset(size_t M, size_t K, int mb, int kb, int m, int k, int pft_dist){
#if L2_LOOP == 1 | L2_LOOP == 2
    // 只要A的遍历顺序是先遍历行，就可以这样算预取地址
    int cur_block_idx = mb * (K/TILE_K) + kb *  (K_CACHE/TILE_K) + k;
    int max_block_idx = ( ( M * K ) / (M_CACHE  * K_CACHE ) ) * (K_CACHE / TILE_K);
    int next_block_idx = mb * (K/TILE_K) + kb *  (K_CACHE/TILE_K) + k + pft_dist;
    next_block_idx = next_block_idx < max_block_idx ? next_block_idx : max_block_idx;
    int mb_next, kb_next, m_next, k_next;
    mb_next = next_block_idx / (K/TILE_K);
    kb_next = ( next_block_idx % (K/TILE_K) ) / (K_CACHE/TILE_K);
    k_next = ( next_block_idx % (K/TILE_K) ) % (K_CACHE/TILE_K);

    size_t pft_off = mb_next  * K * M_CACHE * sizeof(uint8_t) + 
                     kb_next * K_CACHE * M_CACHE * sizeof(uint8_t) +
                     k_next * TILE_K;
    return pft_off;
#else
    return NULL;
#endif
}

size_t B_prefetch_block_offset(size_t K, size_t N, int kb, int nb, int k, int n) {
#if L2_LOOP == 2
    // 外层循环按行遍历
    int cur_block_idx = kb * (N/N_CACHE) * (K_CACHE/TILE_K) + nb * (K_CACHE/TILE_K) + k;
    int max_block_idx = ((K * N) / (K_CACHE * N_CACHE)) * (K_CACHE / TILE_K);
    int next_block_idx = kb * (N/N_CACHE) * (K_CACHE/TILE_K) + nb * (K_CACHE/TILE_K) + k + PFT_DIST;
    next_block_idx = next_block_idx < max_block_idx ? next_block_idx : max_block_idx;
    int kb_next, nb_next, k_next, n_next;
    kb_next = next_block_idx / ((N/N_CACHE)*(K_CACHE/TILE_K));
    nb_next = (next_block_idx % ((K_CACHE/TILE_K)*(N/N_CACHE))) / (K_CACHE/TILE_K);
    k_next = (next_block_idx % ((K_CACHE/TILE_K)*(N/N_CACHE))) % (K_CACHE/TILE_K);

    size_t pft_off = (kb_next*(N/N_CACHE)+nb_next) * (K_CACHE*N_CACHE) * sizeof(uint8_t) +
                        k_next*(TILE_K/4)*(N_CACHE*4);

    return pft_off;
#else
    return NULL;
#endif
}

int get_prefetch_dist() {
    // const char* env = getenv("PFT_DIST");
    // if (env) {
    //     return atoi(env);
    // } else {
    //     return 4;  // 默认值
    // }
    return PFT_DIST;
}

int l2_v2_with_pack_swpfa(int niters, int8_t *A, int8_t *B, int32_t *C, size_t M, size_t N, size_t K) {
    int pft_dist = get_prefetch_dist();
    for (int i = 0; i < niters; ++i) {
#if L2_LOOP == 1
        L2_PACK_IJK_LOOP
#elif L2_LOOP == 2
        L2_PACK_IKJ_LOOP
#else
        L2_PACK_JKI_LOOP
#endif
                    const size_t A_tile_offset = (mb*(K/K_CACHE)+kb) * (M_CACHE*K_CACHE) * sizeof(uint8_t);
                    const size_t B_tile_offset = (kb*(N/N_CACHE)+nb) * (K_CACHE*N_CACHE) * sizeof(uint8_t); 
                    const size_t C_tile_offset = (mb*(N/N_CACHE)+nb) * (M_CACHE*N_CACHE) * sizeof(uint32_t);
                    // const size_t C_tile_offset = 0;

                    for (int m = 0; m < (M_CACHE/TILE_M); m+=2) {
                        for (int n = 0; n < (N_CACHE/TILE_N); n+=2) {
                            if (0 == kb) {
                                _tile_zero(TMM4);
                                _tile_zero(TMM5);
                                _tile_zero(TMM6);
                                _tile_zero(TMM7);
                            } else {
                                _tile_loadd(TMM4, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                                _tile_loadd(TMM5, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                                _tile_loadd(TMM6, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                                _tile_loadd(TMM7, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t)); 
                            }

                            for (int k = 0; k < (K_CACHE/TILE_K); k++) {

                                _tile_loadd(TMM0, A + A_tile_offset + m*TILE_M*K_CACHE+k*TILE_K, K_CACHE);
                                _tile_loadd(TMM1, A + A_tile_offset + (m+1)*TILE_M*K_CACHE+k*TILE_K, K_CACHE);
                                // _tile_loadd(TMM2, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+n*TILE_N*4, N_CACHE*4);
                                // _tile_loadd(TMM3, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+(n+1)*TILE_N*4, N_CACHE*4);
                                _tile_stream_loadd(TMM2, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+n*TILE_N*4, N_CACHE*4);
                                _tile_stream_loadd(TMM3, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+(n+1)*TILE_N*4, N_CACHE*4);

                                // printf("load A in off=%d, ", A_tile_offset + m*TILE_M*K_CACHE+k*TILE_K);
                                // printf("load A in off=%d \n", A_tile_offset + (m+1)*TILE_M*K_CACHE+k*TILE_K);

                                // ======================================TRY1==================================
                                // size_t pft_A_block_off = A_prefetch_block_offset(M, K, mb, kb, m, k, pft_dist);
                                // // printf("%d, %d, %d, %d, %d, %d:  ", mb, kb, nb, m, n, k);
                                // // printf("off=%d \n", pft_A_block_off);
                                // // printf("pft A in off=%d, %d \n", pft_A_block_off, pft_A_block_off + 1*TILE_M*K_CACHE);
                                // prefetch_1024byte_to_l1(A + pft_A_block_off, K_CACHE, 1);
                                // prefetch_1024byte_to_l1(A + pft_A_block_off + 1*TILE_M*K_CACHE, K_CACHE, 1);

                                // ======================================TRY2==================================
                                // if( nb == 0 && (k + pft_dist) < (K_CACHE/TILE_K) ) {
                                //     size_t pft_A_block_off = A_prefetch_block_offset(M, K, mb, kb, m, k, pft_dist);
                                //     // printf("%d, %d, %d, %d, %d, %d:  ", mb, kb, nb, m, n, k);
                                //     // printf("off=%d \n", pft_A_block_off);
                                //     // printf("pft A in off=%d, %d \n", pft_A_block_off, pft_A_block_off + 1*TILE_M*K_CACHE);
                                //     prefetch_1024byte_to_l1(A + pft_A_block_off, K_CACHE, 3);
                                //     prefetch_1024byte_to_l1(A + pft_A_block_off + 1*TILE_M*K_CACHE, K_CACHE, 3);
                                // }

                                //=======================================TRY3==================================
                                if(nb + 1 >= (N/N_CACHE) && (k + pft_dist) >= (K_CACHE/TILE_K)) {
                                    size_t pft_A_block_off = A_prefetch_block_offset(M, K, mb, kb, m, k, pft_dist);
                                    // printf("%d, %d, %d, %d, %d, %d:  ", mb, kb, nb, m, n, k);
                                    // printf("off=%d \n", pft_A_block_off);
                                    // printf("pft A in off=%d, %d \n", pft_A_block_off, pft_A_block_off + 1*TILE_M*K_CACHE);
                                    prefetch_1024byte_to_l1(A + pft_A_block_off, K_CACHE, 3);
                                    prefetch_1024byte_to_l1(A + pft_A_block_off + 1*TILE_M*K_CACHE, K_CACHE, 3);
                                }

                                _tile_dpbuud(TMM4, TMM0, TMM2);
                                _tile_dpbuud(TMM6, TMM1, TMM2);
                                _tile_dpbuud(TMM5, TMM0, TMM3);
                                _tile_dpbuud(TMM7, TMM1, TMM3);
                            }
                            _tile_stored(TMM4, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                            _tile_stored(TMM5, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                            _tile_stored(TMM6, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                            _tile_stored(TMM7, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                        }
                    }  // end of L1 loops

        L2_LOOP_END
    }
}

int l2_v2_with_pack_swpfc(int niters, int8_t *A, int8_t *B, int32_t *C, size_t M, size_t N, size_t K) {
    int pft_dist = 8;
    for (int i = 0; i < niters; ++i) {
#if L2_LOOP == 1
        L2_PACK_IJK_LOOP
#elif L2_LOOP == 2
        L2_PACK_IKJ_LOOP
#else
        L2_PACK_JKI_LOOP
#endif
                    const size_t A_tile_offset = (mb*(K/K_CACHE)+kb) * (M_CACHE*K_CACHE) * sizeof(uint8_t);
                    const size_t B_tile_offset = (kb*(N/N_CACHE)+nb) * (K_CACHE*N_CACHE) * sizeof(uint8_t); 
                    const size_t C_tile_offset = (mb*(N/N_CACHE)+nb) * (M_CACHE*N_CACHE) * sizeof(uint32_t);
                    // const size_t C_tile_offset = 0;

                    for (int m = 0; m < (M_CACHE/TILE_M); m+=2) {
                        for (int n = 0; n < (N_CACHE/TILE_N); n+=2) {
                            if (0 == kb) {
                                _tile_zero(TMM4);
                                _tile_zero(TMM5);
                                _tile_zero(TMM6);
                                _tile_zero(TMM7);
                            } else {
                                _tile_loadd(TMM4, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                                _tile_loadd(TMM5, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                                _tile_loadd(TMM6, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                                _tile_loadd(TMM7, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t)); 

                                // _tile_stream_loadd(TMM4, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                                // _tile_stream_loadd(TMM5, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                                // _tile_stream_loadd(TMM6, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                                // _tile_stream_loadd(TMM7, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t)); 
                            }

                            for (int k = 0; k < (K_CACHE/TILE_K) - pft_dist; k++) {
                                _tile_loadd(TMM0, A + A_tile_offset + m*TILE_M*K_CACHE+k*TILE_K, K_CACHE);
                                _tile_loadd(TMM1, A + A_tile_offset + (m+1)*TILE_M*K_CACHE+k*TILE_K, K_CACHE);
                                // _tile_loadd(TMM2, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+n*TILE_N*4, N_CACHE*4);
                                // _tile_loadd(TMM3, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+(n+1)*TILE_N*4, N_CACHE*4);
                                _tile_stream_loadd(TMM2, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+n*TILE_N*4, N_CACHE*4);
                                _tile_stream_loadd(TMM3, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+(n+1)*TILE_N*4, N_CACHE*4);

                                _tile_dpbuud(TMM4, TMM0, TMM2);
                                _tile_dpbuud(TMM6, TMM1, TMM2);
                                _tile_dpbuud(TMM5, TMM0, TMM3);
                                _tile_dpbuud(TMM7, TMM1, TMM3);
                            }
                            prefetch_1024byte_to_l1((int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t),1);
                            prefetch_1024byte_to_l1((int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t),1);
                            prefetch_1024byte_to_l1((int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t),1);
                            prefetch_1024byte_to_l1((int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t),1);
                            for (int k = (K_CACHE/TILE_K) - pft_dist; k < (K_CACHE/TILE_K); k++) {
                                _tile_loadd(TMM0, A + A_tile_offset + m*TILE_M*K_CACHE+k*TILE_K, K_CACHE);
                                _tile_loadd(TMM1, A + A_tile_offset + (m+1)*TILE_M*K_CACHE+k*TILE_K, K_CACHE);
                                // _tile_loadd(TMM2, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+n*TILE_N*4, N_CACHE*4);
                                // _tile_loadd(TMM3, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+(n+1)*TILE_N*4, N_CACHE*4);
                                _tile_stream_loadd(TMM2, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+n*TILE_N*4, N_CACHE*4);
                                _tile_stream_loadd(TMM3, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+(n+1)*TILE_N*4, N_CACHE*4);

                                _tile_dpbuud(TMM4, TMM0, TMM2);
                                _tile_dpbuud(TMM6, TMM1, TMM2);
                                _tile_dpbuud(TMM5, TMM0, TMM3);
                                _tile_dpbuud(TMM7, TMM1, TMM3);
                            }
                            _tile_stored(TMM4, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                            _tile_stored(TMM5, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                            _tile_stored(TMM6, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                            _tile_stored(TMM7, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                        }
                    }  // end of L1 loops

        L2_LOOP_END
    }
}

int l2_v2_with_pack_swpfb(int niters, int8_t *A, int8_t *B, int32_t *C, size_t M, size_t N, size_t K) {
    // int pft_dist = get_prefetch_dist();
    for (int i = 0; i < niters; ++i) {
#if L2_LOOP == 1
        L2_PACK_IJK_LOOP
#elif L2_LOOP == 2
        L2_PACK_IKJ_LOOP
#else
        L2_PACK_JKI_LOOP
#endif
                    const size_t A_tile_offset = (mb*(K/K_CACHE)+kb) * (M_CACHE*K_CACHE) * sizeof(uint8_t);
                    const size_t B_tile_offset = (kb*(N/N_CACHE)+nb) * (K_CACHE*N_CACHE) * sizeof(uint8_t); 
                    const size_t C_tile_offset = (mb*(N/N_CACHE)+nb) * (M_CACHE*N_CACHE) * sizeof(uint32_t);

                    for (int m = 0; m < (M_CACHE/TILE_M); m+=2) {
                        for (int n = 0; n < (N_CACHE/TILE_N); n+=2) {
                            // if (0 == kb) {
                            //     _tile_zero(TMM4);
                            //     _tile_zero(TMM5);
                            //     _tile_zero(TMM6);
                            //     _tile_zero(TMM7);
                            // } else {
                            //     _tile_loadd(TMM4, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                            //     _tile_loadd(TMM5, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                            //     _tile_loadd(TMM6, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                            //     _tile_loadd(TMM7, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t)); 
                            // }

                            for (int k = 0; k < (K_CACHE/TILE_K); k++) {
                                // _tile_loadd(TMM0, A + A_tile_offset + m*TILE_M*K_CACHE+k*TILE_K, K_CACHE);
                                // _tile_loadd(TMM1, A + A_tile_offset + (m+1)*TILE_M*K_CACHE+k*TILE_K, K_CACHE);
                                // _tile_loadd(TMM2, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+n*TILE_N*4, N_CACHE*4);
                                // _tile_loadd(TMM3, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+(n+1)*TILE_N*4, N_CACHE*4);

                                // _tile_dpbuud(TMM4, TMM0, TMM2);
                                // _tile_dpbuud(TMM6, TMM1, TMM2);

                                // printf("load B in off = %d, %d \n", B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+n*TILE_N*4,
                                //                                  B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+(n+1)*TILE_N*4);

                                size_t pft_B_block_off = B_prefetch_block_offset(K, N, kb, nb, k, n);
                                // printf("%d, %d, %d, %d, %d, %d:  ", mb, kb, nb, m, n, k);
                                // printf("off=%d, %d \n", pft_B_block_off, pft_B_block_off + 1*TILE_N*4);
                                prefetch_1024byte_to_l1(B + pft_B_block_off, N_CACHE*sizeof(uint32_t), 3);
                                prefetch_1024byte_to_l1(B + pft_B_block_off + 1*TILE_N*4, N_CACHE*sizeof(uint32_t), 3);

                                // _tile_dpbuud(TMM5, TMM0, TMM3);
                                // _tile_dpbuud(TMM7, TMM1, TMM3);
                            }
                            // _tile_stored(TMM4, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                            // _tile_stored(TMM5, (int8_t *)C + C_tile_offset + m*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                            // _tile_stored(TMM6, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+n*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                            // _tile_stored(TMM7, (int8_t *)C + C_tile_offset + (m+1)*TILE_M*N_CACHE*sizeof(int32_t)+(n+1)*TILE_N*sizeof(int32_t), N_CACHE * sizeof(uint32_t));
                        }
                    }  // end of L1 loops

        L2_LOOP_END
    }
}

int l2_v2_load_with_pack(int niters, int8_t *A, int8_t *B, int32_t *C, size_t M, size_t N, size_t K) {
    for (int i = 0; i < niters; ++i) {
#if L2_LOOP == 1
        L2_PACK_IJK_LOOP
#elif L2_LOOP == 2
        L2_PACK_IKJ_LOOP
#else
        L2_PACK_JKI_LOOP
#endif
                    const size_t A_tile_offset = (mb*(K/K_CACHE)+kb) * (M_CACHE*K_CACHE) * sizeof(uint8_t);
                    const size_t B_tile_offset = (kb*(N/N_CACHE)+nb) * (K_CACHE*N_CACHE) * sizeof(uint8_t); 
                    const size_t C_tile_offset = (mb*(N/N_CACHE)+nb) * (M_CACHE*N_CACHE) * sizeof(uint32_t);

                    for (int m = 0; m < (M_CACHE/TILE_M); m+=2) {
                        for (int n = 0; n < (N_CACHE/TILE_N); n+=2) {
                            for (int k = 0; k < (K_CACHE/TILE_K); k++) {
                                _tile_loadd(TMM0, A + A_tile_offset + m*TILE_M*K_CACHE+k*TILE_K, K_CACHE);
                                _tile_loadd(TMM1, A + A_tile_offset + (m+1)*TILE_M*K_CACHE+k*TILE_K, K_CACHE);
                                // _tile_loadd(TMM2, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+n*TILE_N*4, N_CACHE*4);
                                // _tile_loadd(TMM3, B + B_tile_offset + k*(TILE_K/4)*(N_CACHE*4)+(n+1)*TILE_N*4, N_CACHE*4);
                            }
                        }
                    }  // end of L1 loops

        L2_LOOP_END
    }
}

void l2_stats(const char* loop_order, const size_t M, const size_t N, const size_t K, 
              const double ops, const double elapsed) {
    double A_tile_bytes = M_CACHE*K_CACHE*sizeof(int8_t);
    double B_tile_bytes = K_CACHE*N_CACHE*sizeof(int8_t);
    double C_tile_bytes = M_CACHE*N_CACHE*sizeof(int8_t);
    double total_tile_bytes = (A_tile_bytes+B_tile_bytes+C_tile_bytes);

    printf("A/B/C/total tile KB:     [%.2f]/[%.2f]/[%.2f]/[%.2f] KB\n", 
           A_tile_bytes/1024, B_tile_bytes/1024, C_tile_bytes/1024, total_tile_bytes/1024);

    int num_A_tiles_to_load = 0, num_B_tiles_to_load = 0, num_C_tiles_to_load = 0, num_C_tiles_to_store = 0;
    if (strcmp(loop_order, "ijk") == 0) {
        num_A_tiles_to_load = (uint32_t)ceil(M / M_CACHE) * (uint32_t)ceil(K / K_CACHE) * (uint32_t)ceil(N / N_CACHE);
        num_B_tiles_to_load = (uint32_t)ceil(N / N_CACHE) * (uint32_t)ceil(K / K_CACHE) * (uint32_t)ceil(M / M_CACHE);
        num_C_tiles_to_load = (uint32_t)ceil(M / M_CACHE) * (uint32_t)ceil(N / N_CACHE);
        num_C_tiles_to_store = (uint32_t)ceil(M / M_CACHE) * (uint32_t)ceil(N / N_CACHE);
    } else if (strcmp(loop_order, "ikj") == 0) {
        num_A_tiles_to_load = (uint32_t)ceil(M / M_CACHE) * (uint32_t)ceil(K / K_CACHE);
        num_B_tiles_to_load = (uint32_t)ceil(N / N_CACHE) * (uint32_t)ceil(K / K_CACHE) * (uint32_t)ceil(M / M_CACHE);
        num_C_tiles_to_load = (uint32_t)ceil(M / M_CACHE) * (uint32_t)ceil(N / N_CACHE) * (uint32_t)ceil(K / K_CACHE);
        num_C_tiles_to_store = (uint32_t)ceil(M / M_CACHE) * (uint32_t)ceil(N / N_CACHE) * (uint32_t)ceil(K / K_CACHE);
    } else if (strcmp(loop_order, "jki") == 0) {
        num_A_tiles_to_load = (uint32_t)ceil(M / M_CACHE) * (uint32_t)ceil(K / K_CACHE) * (uint32_t)ceil(N / N_CACHE);
        num_B_tiles_to_load = (uint32_t)ceil(N / N_CACHE) * (uint32_t)ceil(K / K_CACHE);
        num_C_tiles_to_load = (uint32_t)ceil(M / M_CACHE) * (uint32_t)ceil(N / N_CACHE) * (uint32_t)ceil(K / K_CACHE);
        num_C_tiles_to_store = (uint32_t)ceil(M / M_CACHE) * (uint32_t)ceil(N / N_CACHE) * (uint32_t)ceil(K / K_CACHE);
    }

    double A_load_bytes = num_A_tiles_to_load * A_tile_bytes;
    double B_load_bytes = num_B_tiles_to_load * B_tile_bytes;
    double C_load_bytes = num_C_tiles_to_load * C_tile_bytes;
    double total_load_bytes = A_load_bytes + B_load_bytes + C_load_bytes;

    double C_store_bytes = num_C_tiles_to_store * C_tile_bytes;
    double total_store_bytes = C_store_bytes;

    double theoretical_comp_cycles = ops / 2048;
    double load_bw_req = total_load_bytes / theoretical_comp_cycles;
    double store_bw_req = total_store_bytes / theoretical_comp_cycles;

    printf("Load A Size (L2):     [%.2f] KB\n", A_load_bytes/1024);
    printf("Load B Size (L2):     [%.2f] KB\n", B_load_bytes/1024);
    printf("Load C Size (L2):     [%.2f] KB\n", C_load_bytes/1024);
    printf("Load Size (L2):     [%.2f] KB\n", total_load_bytes/1024);
    printf("Store C Size (L2):     [%.2f] KB\n", C_store_bytes/1024);
    printf("Store Size (L2):     [%.2f] KB\n", total_store_bytes/1024);
    printf("Load BW Req (L2):     [%.2f] bytes/cycle\n", load_bw_req);
    printf("Store BW Req (L2):     [%.2f] bytes/cycle\n", store_bw_req);
}